#include "config.h"

#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <dirent.h>
#include <unistd.h>
#include <time.h>

#define DBG_SUBSYS S_LIBCONTROL

#include "lsv.h"
#include "lsv_bitmap.h"
#include "fileinfo.h"
#include "net_table.h"
#include "table_proto.h"
#include "volume_proto.h"
#include "md_proto.h"
#include "stor_ctl.h"
#include "volume_bh.h"
#include "transaction.h"

static inline int __is_snapshot_in_rollback(table1_t *table1) {

        if (table1->fileinfo.snap_version != table1->fileinfo.snap_rollback) {
                DWARN("file "CHKID_FORMAT" snap %ju %ju\n", CHKID_ARG(&table1->fileinfo.id),
                      table1->fileinfo.snap_version,
                      table1->fileinfo.snap_rollback);
                volume_bh_add(&table1->fileinfo.id, BH_SNAP_ROLLBACK, 0);
                return 1;
        }
        return 0;
}

STATIC int __table1_wrlock(table1_t *table1)
{
        int ret;

        ret = plock_wrlock(&table1->rwlock);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (table1->ltime == 0) {
                ret = EAGAIN;
                GOTO(err_lock, ret);
        }

        return 0;
err_lock:
        plock_unlock(&table1->rwlock);
err_ret:
        return ret;
}

STATIC int __table1_rdlock(table1_t *table1)
{
        int ret;

        ret = plock_rdlock(&table1->rwlock);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (table1->ltime == 0) {
                ret = EAGAIN;
                GOTO(err_lock, ret);
        }

        return 0;
err_lock:
        plock_unlock(&table1->rwlock);
err_ret:
        return ret;
}

STATIC void  __table1_unlock(table1_t *table1)
{
         plock_unlock(&table1->rwlock);
}

STATIC int __table1_extend__(table1_t *table1, int idx)
{
        int ret;

        if (idx >= table1->table_count) {
                ret = yrealloc((void **)&table1->table_array,
                               sizeof(table_proto_t *) * table1->table_count,
                               sizeof(table_proto_t *) * (idx + 1));
                if (unlikely(ret))
                        UNIMPLEMENTED(__DUMP__);

                ret = yrealloc((void **)&table1->vfm_array,
                               sizeof(vfm_mem_t) * table1->table_count,
                               sizeof(vfm_mem_t) * (idx + 1));
                if (unlikely(ret))
                        UNIMPLEMENTED(__DUMP__);

                table1->table_count = idx + 1;
        }

        return 0;
}

/**
 * @todo 对卷的顶级chunk而言，第一个chunk的chkinfo记录在所在的pool里，
 * 余下的记录在第一个chunk的info区的第二段(紧邻fileinfo段)。
 *
 * @param table1
 * @param new_chkinfo
 * @param new_chkstat
 * @return
 */
STATIC int __table1_update_parent__(table1_t *table1, const chkinfo_t *new_chkinfo, const chkstat_t *new_chkstat)
{
        int ret;
        fileid_t parent;
        table_proto_t *table_proto;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;

        const chkid_t *chkid = &new_chkinfo->id;

        YASSERT(chkid->type == __VOLUME_CHUNK__);

        // table_proto = table1->table_proto;
        ret = __table1_get_table_proto_l1(table1, &table_proto, chkid);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        chkstat = table_proto->chkstat;
        chkinfo = table_proto->chkinfo;

        YASSERT(chkid_cmp(&new_chkinfo->id, &chkinfo->id) == 0);

        YASSERT(chkinfo != new_chkinfo);
        YASSERT(chkstat != new_chkstat);
        if (likely(chkinfo->info_version == new_chkinfo->info_version)) {
                memcpy(chkstat, new_chkstat, CHKSTAT_SIZE(new_chkinfo->repnum));
                goto out;
        }

        DINFO("chunk "CHKID_FORMAT" update\n", CHKID_ARG(&chkinfo->id));
        if (chkid->idx == 0) {
                // 先持久化，后更新table proto
                ret = replica_srv_getparent(&new_chkinfo->id, &parent, NULL);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                ret = md_chunk_update(table1->pool, &parent, new_chkinfo,
                                      net_getnid(), chkinfo->info_version);
                if (unlikely(ret)) {
                        GOTO(err_ret, ret);
                }

                __table1_update_table_proto(table1, table_proto, new_chkinfo, new_chkstat);

                //parent need not compare cleanup
                // YASSERT(chkinfo->repnum == new_chkinfo->repnum);
                // memcpy(chkinfo, new_chkinfo, CHKINFO_SIZE(new_chkinfo->repnum));
                // memcpy(chkstat, new_chkstat, CHKSTAT_SIZE(new_chkinfo->repnum));
        } else {
                // TODO ext info

                ret = __table1_update_ext_info(table1, chkid, new_chkinfo, new_chkstat);
                if (unlikely(ret)) {
                        GOTO(err_ret, ret);
                }
        }

out:
        return 0;
err_ret:
        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(&chkinfo->id), ret);
        table1->ltime = 0;
        return ret;
}

STATIC int __table1_get_subvol__(table1_t *table1, table_proto_t **_table,
                                 const chkid_t *tableid)
{
        int ret, idx;
        table_proto_t *table_proto;

        idx = tableid->idx;
        YASSERT(tableid->type == __VOLUME_SUB_CHUNK__);

        if (unlikely(idx >= table1->table_count)) {
                ret = ENOENT;
                GOTO(err_ret, ret);
        }

        if (unlikely(table1->table_array[idx] == NULL)) {
                ret = ENOENT;
                GOTO(err_ret, ret);
        }

        DBUG("table1 get table2 idx:%d tab1 addr:%p tab2 addr:%p \n", idx, table1, table1->table_array[idx]);

        table_proto = table1->table_array[idx];
        *_table = table_proto;

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_check_parent__(table1_t *table1, const fileid_t *parent)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char *buf = mem_cache_calloc1(MEM_CACHE_4K, PAGE_SIZE);
        table_proto_t *table_proto;
        volume_proto_t *volume_proto = table1->volume_proto;

        // TODO ext info
        table_proto = table1->table_proto;

        chkinfo = (void *)buf;
        chkstat = (void *)(buf + PAGE_SIZE / 2);
        CHKINFO_CP(chkinfo, table_proto->chkinfo);
        CHKSTAT_CP(chkstat, table_proto->chkstat, chkinfo->repnum);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_check(table1->pool, chkinfo, chkstat, NULL, parent,
                                    1, &volume_proto->lease.token, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_update_parent__(table1, chkinfo, chkstat);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        mem_cache_free(MEM_CACHE_4K, buf);

        return 0;
err_ret:
        mem_cache_free(MEM_CACHE_4K, buf);
        return ret;
}

/**
 * @param table1
 * @param tid  L2 chkid
 * @param item
 * @return
 */
STATIC int __table1_update_item____(table1_t *table1, const chkid_t *tid, const char *item, int len)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_check_parent__(table1, &table1->parent);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        // TODO L1 ext
        table_proto = table1->table_proto;
        ret = table_proto->update(table_proto, tid->idx, item, len);
        if (unlikely(ret)) {
                DWARN(CHKID_FORMAT" need reload\n",
                      CHKID_ARG(&table_proto->chkid));
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_update_item__(table1_t *table1, const chkinfo_t *_chkinfo,
                                  const chkstat_t *_chkstat)
{
        int ret;
        table_proto_t *table_proto;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char item[FILE_PROTO_ITEM_SIZE];
        const chkid_t *chkid;

        chkid = &_chkinfo->id;
        YASSERT(chkid->idx < FILE_PROTO_EXTERN_ITEM_COUNT * TABLE1_EXT_MAX_NUM);

        ret = __table1_get_subvol__(table1, &table_proto, chkid);
        YASSERT(ret == 0);

        chkstat = table_proto->chkstat;
        chkinfo = table_proto->chkinfo;

        if (chkinfo->info_version == _chkinfo->info_version) {
                memcpy(chkstat, _chkstat, CHKSTAT_SIZE(_chkinfo->repnum));
                goto out;
        }

        DBUG("chunk "CHKID_FORMAT" update\n", CHKID_ARG(&chkinfo->id));
        memcpy(item, _chkinfo, CHKINFO_SIZE(_chkinfo->repnum));

        ret = __table1_update_item____(table1, chkid, item, CHKINFO_SIZE(_chkinfo->repnum));
        if (unlikely(ret)) {
                DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(&chkinfo->id), ret);
                table1->ltime = 0;

                GOTO(err_ret, ret);
        }

        YASSERT(chkid_cmp(&chkinfo->id, chkid) == 0);

        __table1_update_table_proto(table1, table_proto, _chkinfo, _chkstat);


        /* YASSERT(chkinfo->repnum == _chkinfo->repnum); */
        // memcpy(chkinfo, _chkinfo, CHKINFO_SIZE(_chkinfo->repnum));
        // memcpy(chkstat, _chkstat, CHKSTAT_SIZE(_chkinfo->repnum));

out:
        return 0;
err_ret:
        return ret;
}

/**
 *
 * @param table1
 * @param parent
 * @param idx L2 idx
 * @param item
 * @return
 */
int __table1_insert(table1_t *table1, const fileid_t *parent, const chkid_t *tid, const char *item, int len)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_check_parent__(table1, parent);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        // TODO L1 ext
        // table_proto = table1->table_proto;
        uint32_t idx;
        {
                chkid_t top_chkid;
                tid2topid(&top_chkid, tid);
                ret = __table1_get_table_proto_l1(table1, &table_proto, &top_chkid);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                if (top_chkid.idx == 0) {
                        idx = tid->idx;
                } else {
                        YASSERT(tid->idx >= FILE_PROTO_ITEM_COUNT);
                        idx = (tid->idx - FILE_PROTO_ITEM_COUNT) % FILE_PROTO_EXTERN_ITEM_COUNT;
                }
        }

        ret = table_proto->insert(table_proto, idx, item, len);
        if (unlikely(ret)) {
                if (ret == EEXIST) {
                        UNIMPLEMENTED(__WARN__);
                        DWARN(CHKID_FORMAT, CHKID_ARG(&table_proto->chkid));
                } else {
                        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(&table_proto->chkid), ret);
                        table1->ltime = 0;

                        GOTO(err_ret, ret);
                }
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_pre_load(table1_t *table1, const chkid_t *parent,
                             chkinfo_t *chkinfo, chkstat_t *chkstat, int is_ext)
{
        int ret;
        uint64_t info_version;
        volume_proto_t *volume_proto = table1->volume_proto;

        info_version = chkinfo->info_version;

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_check(table1->pool, chkinfo, chkstat, NULL, parent, 1,
                        &volume_proto->lease.token, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        /* maybe chkinfo changed by chunk_proto_check */
        //YASSERT(net_islocal(&chkinfo->diskid[0].id));
        if (!net_islocal(&chkinfo->diskid[0].id) &&
                        chkinfo->id.id == 0 &&                          //vol.xx.1
                        chkinfo->id.type == __VOLUME_CHUNK__) {         //subvol.xx.0
                CHKINFO_DUMP(chkinfo, D_INFO);
                ret = EREMCHG;
                GOTO(err_ret, ret);
        }

        if (info_version != chkinfo->info_version)  {
                DINFO("chunk "CHKID_FORMAT" update\n", CHKID_ARG(&chkinfo->id));
                if (is_ext) {
                        // TODO update L1 ext info
                        //table_proto_t *table_proto = table1->table_proto;
                        //YASSERT(table_proto != NULL);

                        DINFO("update ext info\n");
                        // table_proto->setinfo()
                } else {
                        ret = md_chunk_update(table1->pool, parent, chkinfo, net_getnid(), info_version);
                        if (unlikely(ret))
                                GOTO(err_ret, ret);
                }
        }

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

inline static int __table2_pre_load(table1_t *table1,
                                    chkinfo_t *chkinfo, chkstat_t *chkstat)
{
        int ret;
        uint64_t info_version;
        table_proto_t *table_proto;
        const chkid_t *chkid, *parent;
        char item[FILE_PROTO_ITEM_SIZE];
        volume_proto_t *volume_proto = table1->volume_proto;

        YASSERT(chkinfo->id.type != __RAW_CHUNK__);

        table_proto = table1->table_proto;
        info_version = chkinfo->info_version;
        chkid = &chkinfo->id;
        parent = &table_proto->chkinfo->id;

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_check(table1->pool, chkinfo, chkstat, NULL, parent, 1,
                        &volume_proto->lease.token, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (info_version != chkinfo->info_version)  {
                DBUG("chunk "CHKID_FORMAT" update\n", CHKID_ARG(&chkinfo->id));
                memcpy(item, chkinfo, CHKINFO_SIZE(chkinfo->repnum));
                ret = table_proto->update(table_proto, chkid->idx, item, CHKINFO_SIZE(chkinfo->repnum));
                if (unlikely(ret)) {
                        GOTO(err_ret, ret);
                }
        }

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

/**
 * 设置chkinfo == __chkinfo
 * 清零chkstat
 *
 * @param table1
 * @param __chkinfo
 * @param _table_proto
 * @param new
 * @param volume_proto
 * @return
 */
int __table1_load_table2(table1_t *table1, const chkinfo_t *__chkinfo,
                         table_proto_t **_table_proto, int new, volume_proto_t *volume_proto)
{
        int ret;
        const chkid_t *chkid;
        table_proto_t *table_proto;
        chkstat_t *chkstat;
        chkinfo_t *chkinfo;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];

        (void) new;
        (void) volume_proto;

        chkid = &__chkinfo->id;
        YASSERT((uint64_t)table1->table_count > chkid->idx);

        chkstat = (void *)_chkstat;
        chkinfo = (void *)_chkinfo;
        memset(chkstat, 0x0, CHKSTAT_SIZE(__chkinfo->repnum));
        CHKINFO_CP(chkinfo, __chkinfo);

        ret = table_proto_init(&table_proto, chkinfo, chkstat,
                               FILE_PROTO_EXTERN_ITEM_SIZE,
                               FILE_PROTO_EXTERN_ITEM_COUNT, 0, 0);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        *_table_proto = table_proto;

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_create_table2__(table1_t *table1, const chkid_t *tid, volume_proto_t *volume_proto)
{
        int ret;

        if (tid->idx >= FILE_PROTO_ITEM_COUNT + FILE_PROTO_EXTERN_ITEM_COUNT * TABLE1_EXT_MAX_NUM) {
                ret = EFBIG;
                DWARN("%s too big, idx %d\n", id2str(&volume_proto->chkid), tid->idx);
                GOTO(err_ret, ret);
        }

        ret = __table1_extend__(table1, tid->idx);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_create_table_proto(table1, volume_proto, tid, 1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk____(table1_t *table1, const chkid_t *chkid,
                              chkinfo_t **_chkinfo, chkstat_t **_chkstat,
                              fileid_t *_parent)
{
        int ret;
        table_proto_t *table_proto;
        fileid_t *parent;

        YASSERT(chkid->id == table1->fileinfo.id.id);

        if (chkid->type == __VOLUME_CHUNK__) {
                // TODO ext info
                // table_proto = table1->table_proto;
                ret = __table1_get_table_proto_l1(table1, &table_proto, chkid);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                parent = &table1->parent;
        } else {
                ret = __table1_get_subvol__(table1, &table_proto, chkid);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                parent = &table1->chkid;
        }

        *_chkinfo = table_proto->chkinfo;
        *_chkstat = table_proto->chkstat;

        if (_parent)
                *_parent = *parent;

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk__(table1_t *table1, const chkid_t *chkid,
                            chkinfo_t *_chkinfo, chkstat_t *_chkstat,
                            fileid_t *parent)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        table_proto_t *table_proto;

        if (likely(chkid->id == table1->fileinfo.id.id)) {
                ret = __table1_chunk____(table1, chkid, &chkinfo, &chkstat, parent);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                CHKINFO_CP(_chkinfo, chkinfo);
                if (_chkstat)
                        CHKSTAT_CP(_chkstat, chkstat, chkinfo->repnum);
        } else {
                DBUG("snapshot "CHKID_FORMAT"\n", CHKID_ARG(chkid));

                table_proto = table1->table_proto;

                ret = table_proto->snap->getinfo(table_proto, chkid, _chkinfo);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                YASSERT(_chkstat == NULL);
                YASSERT(parent == NULL);
       }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_update__(table1_t *table1, const chkid_t *chkid, const chkinfo_t *chkinfo,
                             const chkstat_t *chkstat)
{
        int ret;

        YASSERT(chkid->id == table1->fileinfo.id.id);

        if (chkid->type == __VOLUME_CHUNK__) {
                // TODO ext info
                ret = __table1_update_parent__(table1, chkinfo, chkstat);
                if (unlikely(ret)) {
                        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(chkid), ret);
                        table1->ltime = 0;

                        GOTO(err_ret, ret);
                }
        } else {
                ret = __table1_update_item__(table1, chkinfo, chkstat);
                if (unlikely(ret)) {
                        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(chkid), ret);
                        table1->ltime = 0;

                        GOTO(err_ret, ret);
                }
        }

#if 0
        table_proto_t *table_proto;
        DINFO("snapshot "CHKID_FORMAT"\n", CHKID_ARG(chkid));

        table_proto = table1->table_proto;
        ret = table_proto->snap->setinfo(table_proto, chkid, chkinfo);
        if (unlikely(ret))
                GOTO(err_ret, ret);
#endif

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk_set__(table1_t *table1, const chkid_t *chkid,
                                const nid_t *nid, int status)
{
        int ret, seted;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];

        chkinfo = (void *)_chkinfo;
        chkstat = (void *)_chkstat;
        ret = __table1_chunk__(table1, chkid, chkinfo, chkstat, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_set(chkinfo, chkstat, nid, status, &seted);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (seted) {
                ret = __table1_update__(table1, chkid, chkinfo, chkstat);
                if (unlikely(ret))
                        GOTO(err_ret, ret);
        }

        CHKINFO_DUMP(chkinfo, D_INFO);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk_sync__(table1_t *table1, const chkid_t *chkid, int *oflags)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];
        chkid_t parent;
        volume_proto_t *volume_proto = table1->volume_proto;

        DBUG("sync "CHKID_FORMAT"\n", CHKID_ARG(chkid));

        chkinfo = (void *)_chkinfo;
        chkstat = (void *)_chkstat;

        ret = __table1_chunk__(table1, chkid, chkinfo, chkstat, &parent);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        //CHKINFO_DUMP(chkinfo, D_INFO);

        ret = chunk_proto_rep_sync(table1->pool, chkinfo, chkstat, NULL, &parent, 1,
                &volume_proto->lease.token, NULL, oflags);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        SCHEDULE_LEASE_SET();

        ret = __table1_update__(table1, chkid, chkinfo, chkstat);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

int table1_iterator_callback(const void *value, int loc, int idx, void *ctx)
{
        int ret, force = 0;
        volume_proto_t *volume_proto;
        table_proto_t *table_proto;
        const chkinfo_t *chkinfo;
        table1_t *table1;

        (void) loc;
        (void) idx;

        volume_proto = ctx;
        chkinfo = value;
        table1 = &volume_proto->table1;

        ret = table1->extend(table1, chkinfo->id.idx);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        force = table1->fileinfo.attr & __FILE_ATTR_DELETE__;

        // 卷删除过程中,　table1的释放并没有table_proto_del
        // 而是直接chunk_proto_unlink 这个过程中被中断就有可
        // 能造成这里返回错误　其实是这个subvol chunk已经被删
        // 除了,　这样就可以忽略这个chunk 继续删除过程
        ret = __table1_load_table2(table1, chkinfo, &table_proto, 0, volume_proto);
        if (unlikely(ret)) {
                if (force) {
                        goto out;
                } else {
                        GOTO(err_ret, ret);
                }
        }
        volume_proto->table1.table_array[chkinfo->id.idx] = table_proto;

out:
        return 0;
err_ret:
        return ret;
}

STATIC int __table1_load_storage_area(table1_t *table1)
{
        int ret;
        table_proto_t *table_proto;

        table_proto = table1->table_proto;
        ret = table_proto->load(table_proto);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __load_table_proto_l1(table1_t *table1, table_proto_t **_table_proto, const chkid_t *parent,
                                 const chkinfo_t *__chkinfo, int is_ext) {
        int ret;
        table_proto_t *table_proto;
        chkstat_t *chkstat;
        chkinfo_t *chkinfo;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];

        *_table_proto = NULL;

        chkstat = (void *)_chkstat;
        chkinfo = (void *)_chkinfo;
        memset(chkstat, 0x0, CHKSTAT_SIZE(__chkinfo->repnum));
        CHKINFO_CP(chkinfo, __chkinfo);

        ret = __table1_pre_load(table1, parent, chkinfo, chkstat, is_ext);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (is_ext) {
                // 64 * 16000
                ret = table_proto_init(&table_proto, chkinfo, chkstat,
                                       FILE_PROTO_EXTERN_ITEM_SIZE,
                                       FILE_PROTO_EXTERN_ITEM_COUNT, 0, 0);
        } else {
                // 64 * 8000
                ret = table_proto_init(&table_proto, chkinfo, chkstat,
                                       FILE_PROTO_ITEM_SIZE,
                                       FILE_PROTO_ITEM_COUNT, 1, 1);
        }
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table_proto->load(table_proto);
        if (unlikely(ret))
                GOTO(err_free, ret);

        *_table_proto = table_proto;
        return 0;
err_free:
        yfree((void **)&table_proto);
err_ret:
        return ret;
}

int table1_load(table1_t *table1, const chkid_t *parent, const chkinfo_t *__chkinfo)
{
        int ret;
        table_proto_t *table_proto;
        const chkinfo_t *chkinfo;

        ret = __load_table_proto_l1(table1, &table_proto, parent, __chkinfo, 0);
        if (ret)
                GOTO(err_ret, ret);

        // load ext info
        {
                char buf[LICH_BLOCK_SIZE];
                table1_ext_info_t *ext_info;
                int len = LICH_BLOCK_SIZE;
                ret = table_proto->getinfo(table_proto, buf, &len, TABLE_PROTO_BNO_EXT);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                if (len > 0) {
                        ext_info = (table1_ext_info_t *)buf;

                        memcpy(&table1->ext_info, ext_info, sizeof(table1_ext_info_t));

                        // TODO -std=c99
                        int i;
                        for (i=0; i < ext_info->chunk_count; i++) {
                                if (!table1->ext[i])
                                        continue;

                                chkinfo = (const chkinfo_t *)ext_info->chkinfo_array[i];
                                ret = __load_table_proto_l1(table1, &table1->ext[i], parent, chkinfo, 1);
                                if (ret)
                                        GOTO(err_ret, ret);
                        }
                } else {
                        memset(&table1->ext_info, 0, sizeof(table1_ext_info_t));
                }

                DINFO("len %u ext size %u\n", len, table1->ext_info.chunk_count);
        }

        YASSERT(table_proto != NULL);

        table1->table_proto = table_proto;
        ret = __table1_load_storage_area(table1);
        if (ret)
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk_check__(table1_t *table1, const chkid_t *chkid, int op, int *oflags)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];
        chkid_t parent;
        volume_proto_t *volume_proto = table1->volume_proto;

        chkinfo = (void *)_chkinfo;
        chkstat = (void *)_chkstat;
        ret = __table1_chunk__(table1, chkid, chkinfo, chkstat, &parent);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        //CHKINFO_DUMP(chkinfo, D_INFO);
        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_check(table1->pool, chkinfo, chkstat, NULL, &parent, 1,
                        &volume_proto->lease.token, NULL, oflags);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        //CHKINFO_DUMP(chkinfo, D_INFO);
        ret = __table1_update__(table1, chkid, chkinfo, chkstat);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

/**
 *
 * @param table1
 * @param idx table_count
 * @return
 */
STATIC int __table1_extend(table1_t *table1, int idx) {
        int ret;

#if 0
        // extend L1
        if (idx > FILE_PROTO_ITEM_COUNT) {
                int ext_idx = (idx - FILE_PROTO_ITEM_COUNT) / FILE_PROTO_EXTERN_ITEM_COUNT;
                if ((idx - FILE_PROTO_ITEM_COUNT) % FILE_PROTO_EXTERN_ITEM_COUNT != 0) {
                        ext_idx += 1;
                }

                if (ext_idx > table1->ext_info.chunk_count) {
                        // TODO extend
                        DWARN("ext_idx %u %u\n", table1->ext_info.chunk_count, ext_idx);
                }
        }
#endif

        // extend L2
        if (idx >= table1->table_count) {
                ret = __table1_wrlock(table1);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                ret = __table1_extend__(table1, idx);
                if (unlikely(ret))
                        GOTO(err_lock, ret);

                __table1_unlock(table1);
        }

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_get_subvol(table1_t *table1, table_proto_t **_table,
                               const chkid_t *tableid)
{
        int ret;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_get_subvol__(table1, _table, tableid);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_needcheck(table1_t *table1, const chkid_t *chkid,
                              int *_check)
{
        int ret, check1, check2;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;

        YASSERT(chkid->type == __VOLUME_CHUNK__ || chkid->type == __VOLUME_SUB_CHUNK__);

        ret = __table1_rdlock(table1);
        if (unlikely(ret)) {
                YASSERT(ret != ENOENT);
                GOTO(err_ret, ret);
        }

        // check L2
        ret = __table1_chunk____(table1, chkid, &chkinfo, &chkstat, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        check1 = !chunk_proto_consistent(chkinfo, chkstat, NULL);

        // check L1
        ret = __table1_chunk____(table1, &table1->chkid, &chkinfo, &chkstat, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        check2 = !chunk_proto_consistent(chkinfo, chkstat, NULL);

        *_check = check1 || check2;

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

/**
 *
 * @param table1
 * @param chkid vol or subvol chkid
 * @param op
 * @return
 */
STATIC int __table1_chunk_check(table1_t *table1, const chkid_t *chkid, int op, int *oflags)
{
        int ret, check;

        ANALYSIS_BEGIN(0);

        ret = __table1_needcheck(table1, chkid, &check);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (check) {
                if (chkid->type != __VOLUME_SUB_CHUNK__ && chkid->type != __VOLUME_CHUNK__) {
                        ret = EINVAL;
                        GOTO(err_ret, ret);
                }

                ret = __table1_wrlock(table1);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                /* need check parent first, make sure parent chkinfo is all clean */
                // 检查其上一级节点
                if (chkid->type == __VOLUME_SUB_CHUNK__) {
                        // TODO core L1 ext
                        chkid_t top_chkid;
                        tid2topid(&top_chkid, chkid);

                        // check L1 chunk
                        ret = __table1_chunk_check__(table1, &top_chkid, op, oflags);
                        if (unlikely(ret))
                                GOTO(err_lock, ret);
                }

                ret = __table1_chunk_check__(table1, chkid, op, oflags);
                if (unlikely(ret)) {
                        GOTO(err_lock, ret);
                }

                __table1_unlock(table1);
        }

        ANALYSIS_QUEUE(0, IO_WARN, "__table1_chunk_check");

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_create_subvol(table1_t *table1, table_proto_t **_table,
                                  const chkid_t *tableid, void *_volume_proto)
{
        int ret;
        volume_proto_t *volume_proto = _volume_proto;

        YASSERT(tableid->type == __VOLUME_SUB_CHUNK__);

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_ensure_l1(table1, tableid, volume_proto);
        if (unlikely(ret)) {
                GOTO(err_lock, ret);
        }

        {
                // ensure L2
                ret = __table1_get_subvol__(table1, _table, tableid);
                if (ret == 0) {
                        ret = EEXIST;
                        DINFO("table exist\n");
                        GOTO(err_lock, ret);
                }

                ret = __table1_create_table2__(table1, tableid, volume_proto);
                if (unlikely(ret))
                        GOTO(err_lock, ret);

                *_table = table1->table_array[tableid->idx];
        }

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_stat(table1_t *table1, filestat_t *filestat)
{
        int ret, i, tid_count = 0;
        volume_proto_t *volume_proto = table1->volume_proto;
        table2_t *table2 = &volume_proto->table2;
        table_proto_t *table_proto;
        table_proto_t **all_tids;

        filestat->sparse = 0;

        if (table1->table_array == NULL || table1->table_count == 0) {
                DBUG("vol "CHKID_FORMAT"\n", CHKID_ARG(&table1->chkid));
                goto out;
        }

        ret = ymalloc((void **)&all_tids, sizeof(table_proto_t *) * table1->table_count);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        for (i = 0; i < table1->table_count; i++) {
                table_proto = table1->table_array[i];
                if (table_proto != NULL) {
                        all_tids[tid_count++] = table_proto;
                }
        }

        for (i = 0; i < tid_count; i++) {
                table_proto = all_tids[i];
                YASSERT(table_proto != NULL);

                // TODO need to call table2->chunk_check?
                ret = table1->chunk_check(table1, &table_proto->chkid, __OP_WRITE, NULL);
                if (unlikely(ret)) {
                        if (ret == ENOENT) {
                                continue;
                        } else {
                                GOTO(err_free, ret);
                        }
                }

                ret = table2->load_bmap(table2, table_proto);
                if (unlikely(ret)) {
                        DWARN("chkid "CHKID_FORMAT" ret %d\n", CHKID_ARG(&table_proto->chkid), ret);
                        continue;
                }

                filestat->sparse += table_proto->bmap.nr_one;

                DBUG("idx %d chkid "CHKID_FORMAT" bmap %u one %u sparse %u\n",
                     i, CHKID_ARG(&table_proto->chkid), table_proto->bmap.size, table_proto->bmap.nr_one, filestat->sparse);
        }

        yfree((void **)&all_tids);
out:
        return 0;
err_free:
        yfree((void **)&all_tids);
err_ret:
        return ret;
}

STATIC int __table1_chunk_set(table1_t *table1, const chkid_t *chkid,
                              const nid_t *nid, int status, void *_volume_proto)
{
        int ret;

        (void) _volume_proto;

        YASSERT(chkid->type != __RAW_CHUNK__);

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk_set__(table1, chkid, nid, status);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_sync(table1_t *table1, const chkid_t *chkid, void *_volume_proto, int *oflags)
{
        int ret;

        (void) _volume_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk_sync__(table1, chkid, oflags);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_getinfo(table1_t *table1, const chkid_t *chkid,
                                  chkinfo_t *chkinfo)
{
        int ret;

        if (chkid_cmp(&table1->chkid, chkid) == 0)
                YASSERT(chkid->type == __VOLUME_SUB_CHUNK__);

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk__(table1, chkid, chkinfo, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_setattr(table1_t *table1, fileinfo_t *_fileinfo, const setattr_t *setattr)
{
        int ret;
        fileinfo_t fileinfo;
        table_proto_t *table_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        memcpy(&fileinfo, &table1->fileinfo, sizeof(fileinfo_t));

        ret = md_proto_setattr(&fileinfo, setattr, &table_proto->chkinfo->id);
        if (unlikely(ret))
                GOTO(err_lock, ret);

#if LSV
        fileinfo.max_size = VOLUME_SIZE_MAX;
#endif

        // 先持久化，在本地copy上进行，后更新内存部分
        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

        if (_fileinfo)
                memcpy(_fileinfo, &fileinfo, sizeof(fileinfo));

        __table1_unlock(table1);

#if ENABLE_LSV
        if (setattr->size.set_it & SET_LOGICAL_SIZE && IS_RICH_LSV(lich_volume_format(&table1->fileinfo))) {
                ret = lsv_bitmap_resize(&table1->lsv_info, fileinfo.logical_size);
                if (unlikely(ret)) {
                        DERROR("lsv_bitmap_resize error %d\r\n", ret);
                        GOTO(err_ret, ret);
                }
        }
#endif

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

#if 1
STATIC int __table1_attr_force(table1_t *table1)
{
        int ret;
        const nid_t *nid;
        fileinfo_t fileinfo;
        setattr_t setattr;

        DBUG("chunk %s %p\n", id2str(&table1->parent), &table1->parent);
        nid = &table1->parentnid;
        if (net_islocal(nid)) {
                ret = stor_ctl_getattr(&table1->parent, &fileinfo);
                if (unlikely(ret))
                        GOTO(err_ret, ret);
        } else {
                ret = stor_rpc_getattr(nid, &table1->parent, &fileinfo);
                if (unlikely(ret))
                        GOTO(err_ret, ret);
        }

        YASSERT(fileinfo.repnum_usr);
        md_initattr(&setattr, __S_IFREG, fileinfo.repnum_usr);

        ret = __table1_setattr(table1, NULL, &setattr);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_loadattr(table1_t *table1, void *_volume_proto)
{
        int ret, count, table_count, chknum;
        table2_t *table2;
        table_proto_t *table_proto;
        volume_proto_t *volume_proto = _volume_proto;
        char buf[LICH_BLOCK_SIZE];

        table_proto = table1->table_proto;
        count = LICH_BLOCK_SIZE;
        ret = table_proto->getinfo(table_proto, buf,
                                   &count, TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        memcpy(&table1->fileinfo, buf, sizeof(table1->fileinfo));

        DBUG("load "CHKID_FORMAT" size %llu\n",
             CHKID_ARG(&table1->chkid), (LLU)table1->fileinfo.size);

        if (count == 0) {
                DWARN("load empty info "CHKID_FORMAT"\n",
                      CHKID_ARG(&table1->chkid));
                YASSERT(table1->chkid.id);
                ret = __table1_attr_force(table1);
                if (unlikely(ret))
                        GOTO(err_ret, ret);
        }

        YASSERT(table1->fileinfo.repnum_usr <= LICH_REPLICA_MAX);

        // 基于物理空间大小
        // ROW2格式的物理大小处理不同于RAW格式
        // 类似于LSV volume的tail
        chknum = size2chknum(table1->fileinfo.size, &table1->fileinfo.ec);
        table_count = chknum / FILE_PROTO_EXTERN_ITEM_COUNT;
        table_count = (table_count == 0) ? 1 : table_count;

        // extend L1 & L2
        ret = table1->extend(table1, table_count);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        // extend L3
        table2 = &volume_proto->table2;
        ret = table2->extend(table2, chknum, __OP_WRITE);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_getattr(table1_t *table1, fileinfo_t *fileinfo)
{
        int ret;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        *fileinfo = table1->fileinfo;

        DBUG("file "CHKID_FORMAT" size %llu\n", CHKID_ARG(&fileinfo->id), (LLU)fileinfo->size);

        __table1_unlock(table1);

        return 0;
err_ret:
        return ret;
}

#endif

STATIC int __table1_xattr_set(table1_t *table1, const char *key, const char *value,
                        uint32_t valuelen, int flag)
{
        int ret;
        table_proto_t *table_proto;

        (void) valuelen;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->xattr->set( table_proto, key, value, flag);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_xattr_get(table1_t *table1, const char *key, char *value,
                                 int *valuelen)
{
        int ret;
        table_proto_t *table_proto;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->xattr->get( table_proto, key, value);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        *valuelen = strlen(value) + 1;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_xattr_list(table1_t *table1, char *buf, int *buflen)
{
        int ret;
        table_proto_t *table_proto;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->xattr->list(table_proto, buf);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        *buflen = strlen(buf) + 1;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_xattr_remove(table1_t *table1, const char *key)
{
        int ret;
        table_proto_t *table_proto;

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->xattr->remove( table_proto, key);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table1->chunk_check(table1, &table1->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}


STATIC int __table1_move__(table1_t *table1, const chkid_t *chkid, const nid_t *nid, int count)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        char _chkstat[CHKSTAT_MAX], _chkinfo[CHKINFO_MAX];
        chkid_t parent;
        volume_proto_t *volume_proto = table1->volume_proto;

        chkinfo = (void *)_chkinfo;
        chkstat = (void *)_chkstat;
        ret = __table1_chunk__(table1, chkid, chkinfo, chkstat, &parent);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        CHKINFO_DUMP(chkinfo, D_INFO);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_check(table1->pool, chkinfo, chkstat, NULL, &parent, 1,
                &volume_proto->lease.token, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_update__(table1, chkid, chkinfo, chkstat);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = chunk_proto_rep_move(table1->pool, chkinfo, chkstat, NULL, &parent, nid, count,
                &volume_proto->lease.token, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        CHKINFO_DUMP(chkinfo, D_INFO);

        ret = __table1_update__(table1, chkid, chkinfo, chkstat);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = lease_set(&volume_proto->lease);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_table_read_nolock(table1_t *table1, const chkid_t *chkid, buffer_t *buf, size_t size, off_t offset)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;

        YASSERT(chkid_isvolmeta(chkid));
        ret = __table1_chunk____(table1, chkid, &chkinfo, &chkstat, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret =  table_proto_read(chkinfo, chkstat, buf, size, offset, NULL);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_table_read(table1_t *table1, const chkid_t *chkid, buffer_t *buf, size_t size, off_t offset)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk_check__(table1, chkid, __OP_READ, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = __table1_table_read_nolock(table1, chkid, buf, size, offset);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_table_write_nolock(table1_t *table1, const chkid_t *chkid, const buffer_t *buf, size_t size, off_t offset)
{
        int ret;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;

        YASSERT(chkid_isvolmeta(chkid));
        ret = __table1_chunk____(table1, chkid, &chkinfo, &chkstat, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table_proto_write(chkinfo, chkstat, buf, size, offset);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_table_write(table1_t *table1, const chkid_t *chkid, const buffer_t *buf, size_t size, off_t offset)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk_check__(table1, chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = __table1_table_write_nolock(table1, chkid, buf, size, offset);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_move(table1_t *table1, const chkid_t *chkid, const nid_t *nid, int count)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (chkid->type == __VOLUME_CHUNK__ && !net_islocal(&nid[0])) {
                DWARN("table "CHKID_FORMAT" reset\n",  CHKID_ARG(chkid));
        }

        ret = __table1_move__(table1, chkid, nid, count);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_cleanup__(table1_t *table1, const chkid_t *chkid,
                                    const nid_t *nid, uint64_t meta_version)
{
        int ret;
        chkinfo_t *chkinfo;
        char _chkinfo[CHKINFO_MAX];

        DINFO("cleanup "CHKID_FORMAT"\n", CHKID_ARG(chkid));

        chkinfo = (void *)_chkinfo;
        ret = __table1_chunk__(table1, chkid, chkinfo, NULL, NULL);
        if (unlikely(ret)) {
                DWARN("cleanup "CHKID_FORMAT" @ "CHKID_FORMAT" fail\n",
                       CHKID_ARG(chkid), CHKID_ARG(&table1->chkid));

                if (ret == ENOENT) {
                        md_proto_chunk_cleanup1(chkid, nid, meta_version);
                }

                GOTO(err_ret, ret);
        }

        ret = md_proto_chunk_cleanup(chkinfo, nid, meta_version);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk_cleanup(table1_t *table1, const chkid_t *chkid,
                                  const nid_t *nid, uint64_t meta_version, void *volume_proto)
{
        int ret;

        (void) volume_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_chunk_cleanup__(table1, chkid, nid, meta_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_iterator(table1_t *table1, func2_t func2, void *_arg)
{
        int ret, i;
        table_proto_t *table_proto;
        fileid_t fileid;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        DBUG("file "CHKID_FORMAT" count %u\n",
             CHKID_ARG(&table1->chkid), table1->table_count);

        for (i = 0; i < table1->table_count; i++) {
                table_proto = table1->table_array[i];
                if (table_proto) {
                        cid2fid(&fileid, &table_proto->chkinfo->id);
                        func2(_arg, &fileid, table_proto->chkinfo);
                }
        }

        __table1_unlock(table1);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_chunk_unintact(table1_t *table1, func3_t func3, void *_arg)
{
        int ret, i;
        table_proto_t *table_proto;
        fileid_t fileid;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        DBUG("file "CHKID_FORMAT" count %u\n",
             CHKID_ARG(&table1->chkid), table1->table_count);

        for (i = 0; i < table1->table_count; i++) {
                table_proto = table1->table_array[i];
                if (table_proto) {
                        cid2fid(&fileid, &table_proto->chkinfo->id);
                        func3(_arg, &fileid, table_proto->chkinfo, NULL);
                }
        }

        __table1_unlock(table1);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create_dispatch__(table1_t *table1, const char *_site,
                                               chkinfo_t *chkinfo, fileid_t *fileid)
{
        int ret;

        ret = dispatch_newid(fileid);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        fileid->type = __VOLUME_CHUNK__;

        ret = __table1_chunk_newinfo(table1, fileid, chkinfo);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create_table__(table1_t *table1, const char *name, int p, const char *_site)
{
        int ret, offset = 0;
        fileid_t fileid;
        table_proto_t *table_proto;
        chkinfo_t *chkinfo;
        char _chkinfo[CHKINFO_MAX], *data;
        fileinfo_t newinfo;

        // 分配快照头对应的卷
        chkinfo = (void *)_chkinfo;
        ret = __table1_snapshot_create_dispatch__(table1, _site, chkinfo, &fileid);
        if (ret)
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = ymalloc((void **) &data, LICH_CHUNK_SPLIT);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        // 读取并创建快照对应的卷头
        ret = __table_proto_read(table_proto->chkinfo, table_proto->chkstat, data, LICH_CHUNK_SPLIT,
                                  0, NULL);
        if (unlikely(ret)) {
                GOTO(err_free, ret);
        }

        /* just keep TABLE_PROTO_XMAP and TABLE_PROTO_XATTR */
        memset(data + TABLE_PROTO_INFO, 0x0, TABLE_PROTO_INFO_AREA);
        memset(data + TABLE_PROTO_MAP,  0x0, TABLE_PROTO_MAP_AREA);
        memset(data + TABLE_PROTO_SMAP, 0x0, TABLE_PROTO_SMAP_AREA);
        memset(data + TABLE_PROTO_ITEM, 0x0, TABLE_PROTO_ITEM_AREA);
        memset(data + TABLE_PROTO_SNAP, 0x0, TABLE_PROTO_SNAP_AREA);

        //快照都是精简属性,且clone出来的卷也是精简
        char *ptr = data + TABLE_PROTO_XATTR;
        while (offset < TABLE_PROTO_XATTR_AREA) {
                if (strcmp(ptr + offset, LICH_SYSTEM_ATTR_THIN) == 0) {
                        strcpy(ptr + offset + TABLE_PRORO_XATTR_ITEM_SIZE / 2, "thin");
                        break;
                }

                offset += TABLE_PRORO_XATTR_ITEM_SIZE;
        }

        newinfo = table1->fileinfo;
        newinfo.id = fileid;
        YASSERT(newinfo.snap_version == newinfo.snap_rollback);
        newinfo.attr |= __FILE_ATTR_SNAPSHOT__;
        if (p)
                newinfo.attr |= __FILE_ATTR_PROTECT__;

        ret = table_proto_create(table1->pool, chkinfo, &table1->fileinfo.id, net_getnid(),
                                 TABLE_PROTO_HEAD, &newinfo, sizeof(newinfo), data, LICH_CHUNK_SPLIT);
        if (unlikely(ret))
                GOTO(err_free, ret);

        // 登记到卷的快照列表里
        ret = table_proto->snap->create(table_proto, name, chkinfo,
                        table1->fileinfo.snap_version, table1->fileinfo.snap_from);
        if (unlikely(ret))
                GOTO(err_free, ret);

        DINFO("pool %s vol %s snap %s snapid vol.%ju.0 snap_from %jd snap_version %jd\n",
              table1->pool,
              id2str(&table1->chkid),
              name,
              chkinfo->id.id,
              table1->fileinfo.snap_from,
              table1->fileinfo.snap_version);

        yfree((void **)&data);
        return 0;
err_free:
        yfree((void **)&data);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create_setinfo__(table1_t *table1)
{
        int ret;
        table_proto_t *table_proto;
        fileinfo_t fileinfo;
        uint64_t snap_version;

        table_proto = table1->table_proto;
        ret = table_proto->snap->last_version(table_proto, &snap_version);
        if (ret) {
                if (ret == ENOENT) {
                        snap_version = 0;
                } else {
                        GOTO(err_ret, ret);
                }
        }

        snap_version++;

        if (snap_version == (uint64_t) -1) {
                ret = ENOSPC;
                GOTO(err_ret, ret);
        }

        fileinfo = table1->fileinfo;

        YASSERT(fileinfo.snap_version == fileinfo.snap_rollback);

        DINFO("snap_version %jd %jd %jd\n", fileinfo.snap_from, fileinfo.snap_version, snap_version);

        fileinfo.snap_from = fileinfo.snap_version;
        fileinfo.snap_version = snap_version;
        fileinfo.snap_rollback = snap_version;

        YASSERT(fileinfo.snap_version == fileinfo.snap_rollback);

        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table1->fileinfo = fileinfo;

        return 0;
err_ret:
        return ret;
}

int __fix_snap_version(table1_t *table1) {
        int ret;
        table_proto_t *table_proto;
        uint64_t snap_version;

        table_proto = table1->table_proto;
        ret = table_proto->snap->last_version(table_proto, &snap_version);
        if (ret) {
                if (ret == ENOENT) {
                } else {
                        GOTO(err_ret, ret);
                }
        } else {
                if (table1->fileinfo.snap_version == snap_version) {
                        // TODO 修正snap_version
                        DWARN("snap_version %jd %jd\n", table1->fileinfo.snap_version, snap_version);
#if ENABLE_SNAP_FIX_VERSION
                        ret = __table1_snapshot_create_setinfo__(table1);
                        if (ret)
                                GOTO(err_ret, ret);
#else
                        YASSERT(0);
#endif
                } else if (table1->fileinfo.snap_version < snap_version) {
                        YASSERT(0);
                }
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create__(table1_t *table1, const char *name, int p, const char *_site)
{
        int ret;
        table_proto_t *table_proto;

        // 注入故障
        static int __fp = 0;

        DINFO("1: snap %s snap_version %jd %jd\n", name,
              table1->fileinfo.snap_from,
              table1->fileinfo.snap_version);

        table_proto = table1->table_proto;
        ret = table_proto->snap->get_byname(table_proto, name, NULL, NULL);
        if (unlikely(ret)) {
                if (ret == ENOKEY) {
                        ret = __table1_snapshot_create_table__(table1, name, p, _site);
                        if (ret)
                                GOTO(err_ret, ret);

                        if (0 && !__fp) {
                                __fp++;
                                ret = EBUSY;
                                GOTO(err_ret, ret);
                        }

                        // TODO 如果此处发生故障， 会有问题
                        ret = __table1_snapshot_create_setinfo__(table1);
                        if (ret)
                                GOTO(err_ret, ret);

                        DINFO("2: snap %s snap_version %jd %jd\n", name,
                              table1->fileinfo.snap_from,
                              table1->fileinfo.snap_version);
                } else {
                        GOTO(err_ret, ret);
                }
        } else {
                DINFO("vol %s snap %s exists\n", id2str(&table1->chkid), name);
                ret = EEXIST;
                goto err_ret;
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_isempty(table1_t *table1, int *empty)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (ret)
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->is_empty(table_proto, empty);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_check(table1_t *table1, const char *name)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (ret)
                GOTO(err_ret, ret);

        if (__is_snapshot_in_rollback(table1)) {
                ret = EBUSY;
                GOTO(err_lock, ret);
        }

        table_proto = table1->table_proto;
        ret = table_proto->snap->get_byname(table_proto, name, NULL, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create_prep_nolock(table1_t *table1)
{
        int ret, count;
        table_proto_t *table_proto;

        table_proto = table1->table_proto;
        ret = table_proto->snap->count(table_proto, SNAP_USER, &count);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (unlikely(count >= TABLE_PROTO_USER_SNAP_MAX)) {
                ret = EDQUOT;
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_create(table1_t *table1, const char *name, int p, const char *_site, int force)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (__is_snapshot_in_rollback(table1)) {
                ret = EBUSY;
                GOTO(err_lock, ret);
        }

        table_proto = table1->table_proto;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = __table1_snapshot_create_prep_nolock(table1);
        if (unlikely(ret))
                GOTO(err_lock, ret);

#if ENABLE_TRANS_CREATE_SNAPSHOT
        trans_create_snapshot_t new_trans, *old_trans;

        new_trans.base.op = TRANS_CREATE_SNAPSHOT;
        new_trans.base.chkid = table1->chkid;
        new_trans.base.redo = 0;
        strcpy(new_trans.snap, name);
        strcpy(new_trans.site, _site);
        new_trans.priority = p;

        new_trans.snap_from = table1->fileinfo.snap_from;
        new_trans.snap_version = table1->fileinfo.snap_version;

        trans_context_t *ctx;
        ret = trans_begin(&ctx, &new_trans, sizeof(trans_create_snapshot_t));
        if (unlikely(ret)) {
                GOTO(err_lock, ret);
        }

        YASSERT(ctx != NULL);

        // 至此，etcd一定存在事务key

        // 过程必须是可重入的

        // force为真，代表load时的REDO过程
        if (force) {
                YASSERT(!ctx->is_new);

                old_trans = (void *)ctx->buf;

                YASSERT(old_trans->base.op == new_trans.base.op);
                YASSERT(old_trans->priority == new_trans.priority);

                // REDO 检查过程状态，如异常，则修正
                ret = __fix_snap_version(table1);
                if (unlikely(ret))
                        GOTO(err_trans, ret);
        }

        // checklist:
        // - 快照不存在
        // - table1->fileinfo的属性snap_from, snap_version, snap_rollback正确

#endif

        ret = __table1_snapshot_create__(table1, LICH_SYSTEM_ATTR_ROOT, p, _site);
        if (unlikely(ret)) {
                if (ret != EEXIST) {
#if ENABLE_TRANS_CREATE_SNAPSHOT
                        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(&table1->chkid), ret);
                        table1->ltime = 0;
#endif
                        GOTO(err_trans, ret);
                }
        }

        int ret2 = 0;

        ret = __table1_snapshot_create__(table1, name, p, _site);
        if (unlikely(ret)) {
                if (ret == EEXIST) {
                        ret2 = EEXIST;
                } else {
#if ENABLE_TRANS_CREATE_SNAPSHOT
                        DWARN("reset table "CHKID_FORMAT" ret %d\n", CHKID_ARG(&table1->chkid), ret);
                        table1->ltime = 0;
#endif
                        GOTO(err_trans, ret);
                }
        }

        // TODO 故障点

#if ENABLE_TRANS_CREATE_SNAPSHOT
        ret = trans_commit(ctx);
        if (unlikely(ret)) {
                GOTO(err_trans, ret);
        }

        yfree((void **)&ctx);
#endif

        if (!force && unlikely(ret2)) {
                // 不能在事务开始时，检查快照的存在性，因为需要在事务内fix
                ret = EEXIST;
                goto err_lock;
        }

        __table1_unlock(table1);

        return 0;
err_trans:
#if ENABLE_TRANS_CREATE_SNAPSHOT
        yfree((void **)&ctx);
#endif
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_list(table1_t *table1, void *buf, int *len)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->list(table_proto, table1->pool, 0, buf, len);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_iterator(table1_t *table1, func2_t func2, void *_arg)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->iterator(table_proto, func2, _arg);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_unintact(table1_t *table1, func3_t func3, void *_arg)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->unintact(table_proto, func3, _arg);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_iterator2(table1_t *table1, func_int2_t func2, void *_arg)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->iterator2(table_proto, func2, _arg);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_listchild(table1_t *table1, uint64_t snap_from,
                struct list_head *list)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->list_child(table_proto, snap_from, list);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_listparent(table1_t *table1, uint64_t snap_version,
                struct list_head *list)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->list_parent(table_proto, snap_version, list);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_last(table1_t *table1, nid_t *snapnid,
                                  fileid_t *fileid, char *name, uint64_t *snap_version)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->last(table_proto, snapnid, fileid, name, snap_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_update(table1_t *table1,
                                 const chkinfo_t *chkinfo, const nid_t *owner, uint64_t info_version)
{
        int ret;
        const chkid_t *chkid;
        table_proto_t *table_proto;

        (void) owner;
        (void) info_version;
        UNIMPLEMENTED(__WARN__);//check owner;

        chkid = &chkinfo->id;
        YASSERT(chkid_cmp(&table1->chkid, chkid));
        YASSERT(chkid->type == __VOLUME_CHUNK__);

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = table_proto->snap->setinfo(table_proto, &chkinfo->id, chkinfo, info_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_getinfo(table1_t *table1, const fileid_t *fileid, chkinfo_t *chkinfo)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->getinfo(table_proto, fileid, chkinfo);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_lookup(table1_t *table1, const char *name,
                chkinfo_t *chkinfo, uint64_t *snap_version)
{
        int ret;
        const chkid_t *chkid;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->get_byname(table_proto, name, chkinfo, snap_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        chkid = &chkinfo->id;
        YASSERT(chkid_cmp(&table1->chkid, chkid));
        YASSERT(chkid->type == __VOLUME_CHUNK__);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_protect(table1_t *table1, const int on)
{
        int ret;
        fileinfo_t fileinfo;
        table_proto_t *table_proto;

        table_proto = table1->table_proto;
        fileinfo = table1->fileinfo;

        if ((on && (fileinfo.attr & __FILE_ATTR_PROTECT__)) ||
                        (!on && !(fileinfo.attr & __FILE_ATTR_PROTECT__))) {
                goto out;
        } else if (on) {
                fileinfo.attr |= __FILE_ATTR_PROTECT__;
        } else {
                fileinfo.attr &= ~__FILE_ATTR_PROTECT__;
        }

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

        __table1_unlock(table1);

out:
        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rename(table1_t *table1,
                                    const char *name, const char *to)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = table_proto->snap->rename(table_proto, name, to);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_updateparent(table1_t *table1,
                                    const char *name, const uint64_t from)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = table_proto->snap->updateparent(table_proto, name, from);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_setfrom(table1_t *table1, const uint64_t from)
{
        int ret;
        table_proto_t *table_proto;
        fileinfo_t fileinfo;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        fileinfo = table1->fileinfo;

        fileinfo.snap_from = from;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

        __table1_unlock(table1);

        DINFO("table "CHKID_FORMAT" setfrom to %ld\n", CHKID_ARG(&fileinfo.id),
                        fileinfo.snap_from);
        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_remove(table1_t *table1, const char *key)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->remove(table_proto, key);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        DINFO("vol %s snap %s\n", id2str(&table1->chkid), key);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_getversion(table1_t *table1, const chkid_t *chkid, uint64_t *snap_version)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->get_version(table_proto, chkid, snap_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_getbyversion(table1_t *table1, uint64_t snap_version,
                chkinfo_t *chkinfo, char *name)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->get_byversion(table_proto, snap_version, chkinfo, name);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rollback__(table1_t *table1, const char *name)
{
        int ret, has_child, deleting;
        table_proto_t *table_proto;
        uint64_t snap_rollback;
        fileinfo_t fileinfo;
        char auto_name[MAX_NAME_LEN], tmp[MAX_NAME_LEN];
        uuid_t uuid;

        if (__is_snapshot_in_rollback(table1)) {
                ret = EBUSY;
                GOTO(err_ret, ret);
        }

        table_proto = table1->table_proto;
        ret = table_proto->snap->is_deleting(table_proto, &deleting);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (deleting) {
                ret = EBUSY;
                GOTO(err_ret, ret);
        }

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table_proto->snap->get_byname(table_proto, name, NULL, &snap_rollback);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        // TODO fileinfo.snap_from != snap_version且fileinfo.snap_from是叶子节点时，创建auto snap
        if (table1->fileinfo.snap_from != snap_rollback) {
                has_child = table_proto->snap->has_child(table_proto, table1->fileinfo.snap_from);
                if (!has_child) {
                        /* create a auto snapshot to save volume data */
                        uuid_generate(uuid);
                        uuid_unparse(uuid, tmp);
                        sprintf(auto_name, "%s.%s", LICH_SYSTEM_ATTR_AUTO, tmp);
                        ret = __table1_snapshot_create_table__(table1, auto_name, 1, "");
                        if (unlikely(ret))
                                GOTO(err_ret, ret);

                        DINFO("snap_rollback create auto snap %s\n", auto_name);
                }
        }

        // set snap_rollback, entering rollback state
        // 修改副本成功后，更新内存状态
        fileinfo = table1->fileinfo;
        fileinfo.snap_rollback = snap_rollback;

        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

        DINFO("snap_rollback vol %s snap %s snap_from %jd snap_version %jd snap_rollback %jd\n",
              id2str(&table_proto->chkid),
              name,
              fileinfo.snap_from,
              fileinfo.snap_version,
              fileinfo.snap_rollback);

        volume_bh_add(&table1->fileinfo.id, BH_SNAP_ROLLBACK, 0);

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rollback(table1_t *table1, const char *name)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_snapshot_rollback__(table1, name);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_flat__(table1_t *table1, int on)
{
        int ret;
        table_proto_t *table_proto = table1->table_proto;
        fileinfo_t fileinfo = table1->fileinfo;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (on) {
                if (fileinfo.attr & __FILE_ATTR_FLAT__) {
                        DWARN("flat %s\n", id2str(&table1->chkid));
                        ret = EBUSY;
                        GOTO(err_ret, ret);
                }

                // set flat flag on
                fileinfo.attr |= __FILE_ATTR_FLAT__;

                ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo_t), TABLE_PROTO_INFO_ATTR);
                if (unlikely(ret))
                        GOTO(err_ret, ret);

                memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

                volume_bh_add(&table1->fileinfo.id, BH_SNAP_FLAT, 0);
        } else {
                // set clone/flat flag off
                if (fileinfo.attr & __FILE_ATTR_FLAT__ || fileinfo.attr & __FILE_ATTR_CLONE__) {
                        fileinfo.attr &= ~__FILE_ATTR_CLONE__;
                        fileinfo.attr &= ~__FILE_ATTR_FLAT__;


                        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo_t), TABLE_PROTO_INFO_ATTR);
                        if (unlikely(ret))
                                GOTO(err_ret, ret);

                        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));
                }
        }

        DINFO("snap flat vol %s on %d\n", id2str(&table1->chkid), on);
        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_flat(table1_t *table1, int on)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_snapshot_flat__(table1, on);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_next(table1_t *table1, const chkid_t *chkid, chkid_t *next, char *name, uint64_t *snap_version)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->next(table_proto, chkid, next, name, snap_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_prev(table1_t *table1, const chkid_t *chkid,
                chkid_t *prev, char *name, uint64_t *snap_version)
{
        int ret;
        table_proto_t *table_proto;

        ret = __table1_rdlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = table_proto->snap->prev(table_proto, chkid, prev, name, snap_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rollback_table__(table1_t *table1, buffer_t *buf)
{
        int ret;
        table_proto_t *table_proto;
        char *data;

        table_proto = table1->table_proto;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = ymalloc((void **)&data, LICH_CHUNK_SPLIT);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        mbuffer_get(buf, data, buf->len);

        ret = __table_proto_write(table_proto->chkinfo, table_proto->chkstat,
                        data + TABLE_PROTO_XATTR, TABLE_PROTO_XATTR_AREA, TABLE_PROTO_XATTR);
        if (unlikely(ret))
                GOTO(err_free, ret);

        ret = __table_proto_write(table_proto->chkinfo, table_proto->chkstat,
                        data + TABLE_PROTO_XMAP, TABLE_PROTO_XMAP_AREA, TABLE_PROTO_XMAP);
        if (unlikely(ret))
                GOTO(err_free, ret);

        yfree((void **)&data);
        return 0;
err_free:
        yfree((void **)&data);
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rollback_setinfo__(table1_t *table1)
{
        int ret;
        uint64_t snap_version;
        table_proto_t *table_proto;
        fileinfo_t fileinfo;

        table_proto = table1->table_proto;
        fileinfo = table1->fileinfo;

        DINFO("rollback "CHKID_FORMAT" %ju --> %ju\n", CHKID_ARG(&fileinfo.id),
              fileinfo.snap_version, fileinfo.snap_rollback);

        ret = table_proto->snap->last_version(table_proto, &snap_version);
        if (ret) {
                if (ret == ENOENT) {
                        /* should not come here */
                        YASSERT(0);
                } else {
                        GOTO(err_ret, ret);
                }
        }

        fileinfo.snap_from = fileinfo.snap_rollback;
        fileinfo.snap_version = ++snap_version;
        fileinfo.snap_rollback = fileinfo.snap_version;

        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = table_proto->setinfo(table_proto, &fileinfo, sizeof(fileinfo), TABLE_PROTO_INFO_ATTR);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        memcpy(&table1->fileinfo, &fileinfo, sizeof(fileinfo_t));

        return 0;
err_ret:
        return ret;
}

STATIC int __table1_snapshot_rollback_bh__(table1_t *table1, buffer_t *buf)
{
        int ret;

        ret = __table1_snapshot_rollback_table__(table1, buf);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_snapshot_rollback_setinfo__(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}


STATIC int __table1_snapshot_rollback_bh(table1_t *table1, buffer_t *buf)
{
        int ret;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __table1_snapshot_rollback_bh__(table1, buf);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

STATIC int __table1_chunk_reject(table1_t *table1,  const chkid_t *chkid,
                                 const nid_t *bad, chkinfo_t *chkinfo)
{
        int ret;
        table_proto_t *table_proto;
        uint64_t info_version;

        ret = __table1_wrlock(table1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table_proto = table1->table_proto;
        ret = __table1_chunk_check__(table1, &table_proto->chkid, __OP_WRITE, NULL);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        ret = table_proto->snap->getinfo(table_proto, chkid, chkinfo);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        info_version = chkinfo->info_version;
        ret = md_proto_reject(chkinfo, bad);
        if (unlikely(ret)) {
                GOTO(err_lock, ret);
        }

        ret = table_proto->snap->setinfo(table_proto, chkid, chkinfo, info_version);
        if (unlikely(ret))
                GOTO(err_lock, ret);

        CHKINFO_DUMP(chkinfo, D_INFO);

        __table1_unlock(table1);

        return 0;
err_lock:
        __table1_unlock(table1);
err_ret:
        return ret;
}

int __table1_vfm_stat(table1_t *table1, int *count)
{
        int ret, i, tid_count = 0;
        volume_proto_t *volume_proto = table1->volume_proto;
        table2_t *table2 = &volume_proto->table2;
        table_proto_t *table_proto;
        chkid_t *all_tids;
        chkid_t chkid, tid;
        char _vfm[MAX_BUF_LEN];
        vfm_t *vfm = (void *)_vfm;

        *count = 0;

        if (table1->table_array == NULL || table1->table_count == 0) {
                DBUG("vol "CHKID_FORMAT"\n", CHKID_ARG(&table1->chkid));
                goto out;
        }

        ret = ymalloc((void **)&all_tids, sizeof(chkid_t) * table1->table_count);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        for (i = 0; i < table1->table_count; i++) {
                table_proto = table1->table_array[i];
                if (table_proto != NULL) {
                        all_tids[tid_count++] = table_proto->chkid;
                }
        }

        for (i = 0; i < tid_count; i++) {
                tid = all_tids[i];

                ret = table1->chunk_check(table1, &tid, __OP_WRITE, NULL);
                if (unlikely(ret)) {
                        DWARN(""CHKID_FORMAT" ret %d\n", CHKID_ARG(&tid), ret);
                        continue;
                }

                tid2cid(&chkid, &tid, 0);

                ret = table2->vfm_get(table2, &chkid, vfm);
                if (unlikely(ret)) {
                        if (ret == ENOKEY) {
                                DBUG(""CHKID_FORMAT" ret %d\n", CHKID_ARG(&chkid), ret);
                        } else {
                                DWARN(""CHKID_FORMAT" ret %d\n", CHKID_ARG(&chkid), ret);
                        }
                        continue;
                }

                DBUG(""CHKID_FORMAT" count %d\n", CHKID_ARG(&chkid), vfm->count);
                if (vfm && vfm->count > 0) {
                        *count += 1;
                }
        }

        yfree((void **)&all_tids);

out:
        DBUG("pool %s vol "CHKID_FORMAT" vfm %d\n", table1->pool, CHKID_ARG(&table1->chkid), *count);
        return 0;
err_ret:
        return ret;
}

int table1_init(table1_t *table1, const char *pool, const fileid_t *fileid,
                const fileid_t *parent, void *volume_proto)
{
        int ret;
        char *pname = NULL;
#if LOCK_DEBUG
        char lname[MAX_LOCK_NAME];

        pname = lname;
        sprintf(pname, "table1."CHKID_FORMAT, CHKID_ARG(fileid));
#endif

        memset(table1, 0x0, sizeof(*table1));

        strcpy(table1->pool, pool);

        ret = plock_init(&table1->rwlock, pname);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        table1->stat = __table1_stat;

        table1->extend = __table1_extend;

        table1->create_table2 = __table1_create_subvol;

        table1->get_table2 = __table1_get_subvol;
        table1->get_table2_nolock = __table1_get_subvol__;

        table1->chunk_getinfo = __table1_chunk_getinfo;
        table1->chunk_iterator = __table1_chunk_iterator;
        table1->chunk_unintact = __table1_chunk_unintact;

        table1->chunk_set = __table1_chunk_set;
        table1->chunk_check = __table1_chunk_check;
        table1->chunk_sync = __table1_chunk_sync;
        table1->move = __table1_move;
        table1->chunk_update = __table1_chunk_update;
        table1->chunk_reject = __table1_chunk_reject;
        table1->chunk_cleanup = __table1_chunk_cleanup;

        table1->table_read = __table1_table_read;
        table1->table_write = __table1_table_write;

        table1->loadattr = __table1_loadattr;
        table1->getattr = __table1_getattr;
        table1->setattr = __table1_setattr;

        table1->xattr_set = __table1_xattr_set;
        table1->xattr_get = __table1_xattr_get;
        table1->xattr_list = __table1_xattr_list;
        table1->xattr_remove = __table1_xattr_remove;

        table1->snapshot_create = __table1_snapshot_create;
        table1->snapshot_remove = __table1_snapshot_remove;
        table1->snapshot_isempty = __table1_snapshot_isempty;
        table1->snapshot_check = __table1_snapshot_check;
        table1->snapshot_list = __table1_snapshot_list;
        table1->snapshot_iterator = __table1_snapshot_iterator;
        table1->snapshot_unintact = __table1_snapshot_unintact;
        table1->snapshot_iterator2 = __table1_snapshot_iterator2;
        table1->snapshot_listchild = __table1_snapshot_listchild;
        table1->snapshot_listparent = __table1_snapshot_listparent;
        table1->snapshot_last = __table1_snapshot_last;
        table1->snapshot_getinfo = __table1_snapshot_getinfo;
        table1->snapshot_lookup = __table1_snapshot_lookup;
        table1->snapshot_protect = __table1_snapshot_protect;
        table1->snapshot_rename = __table1_snapshot_rename;
        table1->snapshot_updateparent = __table1_snapshot_updateparent;
        table1->snapshot_setfrom = __table1_snapshot_setfrom;
        table1->snapshot_getversion = __table1_snapshot_getversion;
        table1->snapshot_getbyversion = __table1_snapshot_getbyversion;
        table1->snapshot_next = __table1_snapshot_next;
        table1->snapshot_prev = __table1_snapshot_prev;

        table1->snapshot_rollback = __table1_snapshot_rollback;
        table1->snapshot_rollback_bh = __table1_snapshot_rollback_bh;

        table1->snapshot_flat = __table1_snapshot_flat;

        table1->vfm_stat = __table1_vfm_stat;

        //
        table1->wrlock = __table1_wrlock;
        table1->rdlock = __table1_rdlock;
        table1->unlock = __table1_unlock;

        table1->chkid = *fileid;
        table1->parent = *parent;
        table1->volume_proto = volume_proto;

        return 0;
err_ret:
        return ret;
}

void table1_destroy(table1_t *table1)
{
        int i;
        table_proto_t *table_proto;

        if (table1 == NULL)
                return ;

        if (table1->table_proto) {
                table_proto_destroy(table1->table_proto);
                table1->table_proto = NULL;
        }

        if (table1->table_array) {
                for (i = 0; i < table1->table_count; i++) {
                        table_proto = table1->table_array[i];
                        if (table_proto) {
                                table_proto_destroy(table_proto);
                        }

                        if (table1->vfm_array[i].vfm) {
                                yfree((void **)&table1->vfm_array[i].vfm);
                                table1->vfm_array[i].vfm = NULL;
                        }
                }

                yfree((void **)&table1->table_array);
                table1->table_array = NULL;
                table1->table_count = 0;
        }
}

int table1_cleanup(table1_t *table1)
{
        int ret, i;
        chkinfo_t *chkinfo;
        chkstat_t *chkstat;
        table_proto_t *table_proto;

        if (table1->table_array) {
                for (i = 0; i < table1->table_count; i++) {
                        table_proto = table1->table_array[i];
                        if (table_proto) {
                                chkinfo = table_proto->chkinfo;
                                chkstat = table_proto->chkstat;

                                DINFO("table1 "CHKID_FORMAT" unlink\n", CHKID_ARG(&chkinfo->id));
                                ret = chunk_proto_rep_unlink(chkinfo, chkstat);
                                if (unlikely(ret)) {
                                        if (ret == ENOENT)
                                                continue;
                                        else
                                                GOTO(err_ret, ret);
                                }
                        }
                }
        }

        // TODO L1 ext
        table_proto = table1->table_proto;
        if (table_proto) {
                chkinfo = table_proto->chkinfo;
                chkstat = table_proto->chkstat;
                ret = chunk_proto_rep_unlink(chkinfo, chkstat);
                if (unlikely(ret)) {
                        if (ret != ENOENT)
                                GOTO(err_ret, ret);
                }
        }

        return 0;
err_ret:
        return ret;
}
